{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "<img src=\"cnn_vae.png\" width=\"300\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "PARAMS = {\n",
    "    'max_len': 15,\n",
    "    'embed_dims': 128,\n",
    "    'rnn_size': 128,\n",
    "    'cnn_size': 128,\n",
    "    'latent_size': 16,\n",
    "    'kernel_sz': 3,\n",
    "    'n_hidden_layer': 3,\n",
    "    'num_sampled': 1000,\n",
    "    'clip_norm': 5.0,\n",
    "    'anneal_max': 1.0,\n",
    "    'anneal_bias': 6000,\n",
    "    'batch_size': 128,\n",
    "    'n_epochs': 10,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_vocab(index_from=4):\n",
    "    PARAMS['word2idx'] = tf.keras.datasets.imdb.get_word_index()\n",
    "    PARAMS['word2idx'] = {k: (v + index_from) for k, v in PARAMS['word2idx'].items()}\n",
    "    PARAMS['word2idx']['<pad>'] = 0\n",
    "    PARAMS['word2idx']['<start>'] = 1\n",
    "    PARAMS['word2idx']['<unk>'] = 2\n",
    "    PARAMS['word2idx']['<end>'] = 3\n",
    "    \n",
    "    PARAMS['idx2word'] = {i: w for w, i in PARAMS['word2idx'].items()}\n",
    "    \n",
    "    PARAMS['vocab_size'] = len(PARAMS['word2idx'])\n",
    "\n",
    "    \n",
    "def load_data(index_from=4):\n",
    "    (X_train, _), (X_test, _) = tf.contrib.keras.datasets.imdb.load_data(\n",
    "        num_words=None, index_from=index_from)\n",
    "    return (X_train, X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "word2idx = build_vocab()\n",
    "X = np.concatenate(load_data())\n",
    "\n",
    "X = np.concatenate((\n",
    "    tf.keras.preprocessing.sequence.pad_sequences(\n",
    "        X, PARAMS['max_len'], truncating='post', padding='post'),\n",
    "    tf.keras.preprocessing.sequence.pad_sequences(\n",
    "        X, PARAMS['max_len'], truncating='pre', padding='post')))\n",
    "\n",
    "enc_inp = X[:, 1:]\n",
    "dec_inp = X\n",
    "dec_out = np.concatenate([X[:, 1:], np.full([X.shape[0], 1], PARAMS['word2idx']['<end>'])], 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def reparam_trick(z_mean, z_logvar):\n",
    "    gaussian = tf.truncated_normal(tf.shape(z_logvar))\n",
    "    z = z_mean + tf.exp(0.5 * z_logvar) * gaussian\n",
    "    return z\n",
    "\n",
    "\n",
    "def kl_w_fn(global_step):\n",
    "    return PARAMS['anneal_max'] * tf.sigmoid((10 / PARAMS['anneal_bias']) * (\n",
    "        tf.to_float(global_step) - tf.constant(PARAMS['anneal_bias'] / 2)))\n",
    "\n",
    "\n",
    "def kl_loss_fn(mean, gamma):\n",
    "    return 0.5 * tf.reduce_sum(\n",
    "        tf.exp(gamma) + tf.square(mean) - 1 - gamma) / tf.to_float(tf.shape(mean)[0])\n",
    "\n",
    "\n",
    "def clip_grads(loss):\n",
    "    variables = tf.trainable_variables()\n",
    "    grads = tf.gradients(loss, variables)\n",
    "    clipped_grads, _ = tf.clip_by_global_norm(grads, PARAMS['clip_norm'])\n",
    "    return zip(clipped_grads, variables)\n",
    "\n",
    "\n",
    "def rnn_cell():\n",
    "    return tf.nn.rnn_cell.GRUCell(PARAMS['rnn_size'],\n",
    "                                  kernel_initializer=tf.orthogonal_initializer())\n",
    "\n",
    "\n",
    "def cnn_block(x, dilation_rate, pad_sz):\n",
    "    pad = tf.zeros([tf.shape(x)[0], pad_sz, x.get_shape()[-1].value])\n",
    "    x =  tf.layers.conv1d(inputs = tf.concat([pad, x, pad], 1),\n",
    "                          filters = PARAMS['cnn_size'],\n",
    "                          kernel_size = PARAMS['kernel_sz'],\n",
    "                          dilation_rate = dilation_rate)\n",
    "    x = x[:, :-pad_sz, :]\n",
    "    x = tf.nn.relu(x)\n",
    "    return x\n",
    "\n",
    "\n",
    "def cnn_forward(x, output_proj):\n",
    "    for i in range(PARAMS['n_hidden_layer']):\n",
    "        dilation_rate = 2 ** i\n",
    "        pad_sz = (PARAMS['kernel_sz'] - 1) * dilation_rate\n",
    "        x += cnn_block(x, dilation_rate, pad_sz)\n",
    "        logits = output_proj(x)\n",
    "    return logits, x\n",
    "\n",
    "\n",
    "def autoregressive(embedding, z, input_proj, output_proj):\n",
    "    batch_sz = tf.shape(z)[0]\n",
    "    \n",
    "    def cond(i, x, temp):\n",
    "        return i < PARAMS['max_len']\n",
    "\n",
    "    def body(i, x, temp):\n",
    "        sos = tf.fill([batch_sz, 1], PARAMS['word2idx']['<start>'])\n",
    "        x = tf.concat([sos, x[:, :-1]], 1)\n",
    "        \n",
    "        x = tf.nn.embedding_lookup(embedding, x)\n",
    "        x = input_proj(tf.concat([x, z], -1))\n",
    "        logits = cnn_forward(x, output_proj)[0]\n",
    "        ids = tf.argmax(logits, -1, output_type=tf.int32)[:, i]\n",
    "        ids = tf.expand_dims(ids, -1)\n",
    "\n",
    "        temp = tf.concat([temp[:, 1:], ids], -1)\n",
    "\n",
    "        x = tf.concat([temp[:, -(i+1):], temp[:, :-(i+1)]], -1)\n",
    "        x = tf.reshape(x, [batch_sz, PARAMS['max_len']])\n",
    "        i += 1\n",
    "        return i, x, temp\n",
    "    \n",
    "    x = tf.zeros([batch_sz, PARAMS['max_len']], tf.int32)\n",
    "    _, res, _ = tf.while_loop(cond, body, [tf.constant(0), x, x])\n",
    "    \n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(inputs, labels, mode):\n",
    "    is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n",
    "    enc_seq_len = tf.count_nonzero(inputs, 1, dtype=tf.int32)\n",
    "    batch_sz = tf.shape(inputs)[0]\n",
    "    \n",
    "    with tf.variable_scope('Encoder'):\n",
    "        embedding = tf.get_variable('lookup_table', [len(PARAMS['word2idx']), PARAMS['embed_dims']])\n",
    "        x = tf.nn.embedding_lookup(embedding, inputs)\n",
    "        \n",
    "        _, enc_state = tf.nn.dynamic_rnn(rnn_cell(), x, enc_seq_len, dtype=tf.float32)\n",
    "        \n",
    "        z_mean = tf.layers.dense(enc_state, PARAMS['latent_size'])\n",
    "        z_logvar = tf.layers.dense(enc_state, PARAMS['latent_size'])\n",
    "        \n",
    "    z = reparam_trick(z_mean, z_logvar)\n",
    "        \n",
    "    with tf.variable_scope('Decoder'):\n",
    "        input_proj = tf.layers.Dense(PARAMS['cnn_size'], tf.nn.relu)\n",
    "        output_proj = tf.layers.Dense(len(PARAMS['word2idx']), _scope='decoder/output_proj')\n",
    "        z = tf.tile(tf.expand_dims(z, 1), [1, PARAMS['max_len'], 1])\n",
    "        \n",
    "        if is_training:\n",
    "            dec_inp = tf.nn.embedding_lookup(embedding, labels['dec_inp'])\n",
    "            x = input_proj(tf.concat([dec_inp, z], -1))\n",
    "            logits, x = cnn_forward(x, output_proj)\n",
    "            return x, logits, (z_mean, z_logvar)\n",
    "        else:\n",
    "            return autoregressive(embedding, z, input_proj, output_proj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_fn(features, labels, mode):\n",
    "    logits_or_ids = forward(features, labels, mode)        \n",
    "    \n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(mode, predictions=logits_or_ids)\n",
    "        \n",
    "    if mode == tf.estimator.ModeKeys.TRAIN:\n",
    "        cnn_output, logits, (z_mean, z_logvar) = logits_or_ids\n",
    "        \n",
    "        global_step = tf.train.get_global_step()\n",
    "        \n",
    "        with tf.variable_scope('Decoder/decoder/output_proj', reuse=True):\n",
    "            _weights = tf.transpose(tf.get_variable('kernel'))\n",
    "            _biases = tf.get_variable('bias')\n",
    "        \n",
    "        mask = tf.reshape(tf.to_float(tf.sign(labels['dec_out'])), [-1])\n",
    "        \n",
    "        nll_loss = tf.reduce_sum(mask * tf.nn.sampled_softmax_loss(\n",
    "            weights = _weights,\n",
    "            biases = _biases,\n",
    "            labels = tf.reshape(labels['dec_out'], [-1, 1]),\n",
    "            inputs = tf.reshape(cnn_output, [-1, PARAMS['cnn_size']]),\n",
    "            num_sampled = PARAMS['num_sampled'],\n",
    "            num_classes = PARAMS['vocab_size'],\n",
    "        )) / tf.to_float(tf.shape(features)[0])\n",
    "        \n",
    "        kl_w = kl_w_fn(global_step)\n",
    "        \n",
    "        kl_loss = kl_loss_fn(z_mean, z_logvar)\n",
    "        \n",
    "        loss_op = nll_loss + kl_w * kl_loss\n",
    "        \n",
    "        train_op = tf.train.AdamOptimizer().apply_gradients(\n",
    "            clip_grads(loss_op),\n",
    "            global_step = global_step)\n",
    "        \n",
    "        lth = tf.train.LoggingTensorHook(\n",
    "            {'nll_loss': nll_loss, 'kl_w': kl_w, 'kl_loss': kl_loss}, every_n_iter=100)\n",
    "        \n",
    "        return tf.estimator.EstimatorSpec(\n",
    "            mode=mode, loss=loss_op, train_op=train_op, training_hooks=[lth])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': None, '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x1154a7710>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:1344: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 156.41869, step = 1\n",
      "INFO:tensorflow:nll_loss = 156.41869, kl_w = 0.006692851, kl_loss = 4.612839e-05\n",
      "INFO:tensorflow:global_step/sec: 2.79942\n",
      "INFO:tensorflow:loss = 85.66326, step = 101 (35.723 sec)\n",
      "INFO:tensorflow:nll_loss = 85.66301, kl_w = 0.007897083, kl_loss = 0.032010667 (35.722 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.85468\n",
      "INFO:tensorflow:loss = 82.23248, step = 201 (35.030 sec)\n",
      "INFO:tensorflow:nll_loss = 82.20407, kl_w = 0.009315956, kl_loss = 3.049909 (35.030 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.94169\n",
      "INFO:tensorflow:loss = 76.12364, step = 301 (33.995 sec)\n",
      "INFO:tensorflow:nll_loss = 76.02651, kl_w = 0.010986943, kl_loss = 8.840496 (33.995 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.80931\n",
      "INFO:tensorflow:loss = 69.686066, step = 401 (35.595 sec)\n",
      "INFO:tensorflow:nll_loss = 69.47872, kl_w = 0.012953726, kl_loss = 16.006754 (35.595 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.79791\n",
      "INFO:tensorflow:loss = 71.9909, step = 501 (35.741 sec)\n",
      "INFO:tensorflow:nll_loss = 71.706474, kl_w = 0.015267149, kl_loss = 18.629833 (35.741 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.86229\n",
      "INFO:tensorflow:loss = 68.61781, step = 601 (34.937 sec)\n",
      "INFO:tensorflow:nll_loss = 68.24541, kl_w = 0.01798621, kl_loss = 20.705296 (34.937 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.85821\n",
      "INFO:tensorflow:loss = 67.446396, step = 701 (34.987 sec)\n",
      "INFO:tensorflow:nll_loss = 66.94687, kl_w = 0.021179108, kl_loss = 23.585876 (34.987 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 782 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 66.44753.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: it was not see it was a lot of harp harp harp harp harp <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a good movie but harp harp harp harp harp it is harp <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-782\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 783 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 67.70235, step = 783\n",
      "INFO:tensorflow:nll_loss = 67.08449, kl_w = 0.024205629, kl_loss = 25.525494\n",
      "INFO:tensorflow:global_step/sec: 2.88768\n",
      "INFO:tensorflow:loss = 63.027763, step = 883 (34.631 sec)\n",
      "INFO:tensorflow:nll_loss = 62.283806, kl_w = 0.028470589, kl_loss = 26.13078 (34.631 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.97785\n",
      "INFO:tensorflow:loss = 63.538166, step = 983 (33.581 sec)\n",
      "INFO:tensorflow:nll_loss = 62.651848, kl_w = 0.033461247, kl_loss = 26.487953 (33.581 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.04253\n",
      "INFO:tensorflow:loss = 60.650826, step = 1083 (32.867 sec)\n",
      "INFO:tensorflow:nll_loss = 59.567413, kl_w = 0.039291352, kl_loss = 27.573818 (32.868 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.00996\n",
      "INFO:tensorflow:loss = 57.863464, step = 1183 (33.223 sec)\n",
      "INFO:tensorflow:nll_loss = 56.627083, kl_w = 0.04608883, kl_loss = 26.826088 (33.223 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.75289\n",
      "INFO:tensorflow:loss = 59.67399, step = 1283 (36.325 sec)\n",
      "INFO:tensorflow:nll_loss = 58.28183, kl_w = 0.053996176, kl_loss = 25.782528 (36.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.87727\n",
      "INFO:tensorflow:loss = 58.643364, step = 1383 (34.755 sec)\n",
      "INFO:tensorflow:nll_loss = 57.013912, kl_w = 0.06317033, kl_loss = 25.794586 (34.755 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.84838\n",
      "INFO:tensorflow:loss = 56.533566, step = 1483 (35.108 sec)\n",
      "INFO:tensorflow:nll_loss = 54.60893, kl_w = 0.07378165, kl_loss = 26.085556 (35.108 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 1564 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 56.104996.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-1564\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: so many people know this film is so much because of the movie patient' <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a good movie about me in all i was disappointed about <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-1564\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 1565 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 50.531075, step = 1565\n",
      "INFO:tensorflow:nll_loss = 48.336384, kl_w = 0.08368247, kl_loss = 26.22642\n",
      "INFO:tensorflow:global_step/sec: 2.88687\n",
      "INFO:tensorflow:loss = 51.889847, step = 1665 (34.641 sec)\n",
      "INFO:tensorflow:nll_loss = 49.406578, kl_w = 0.09738124, kl_loss = 25.50048 (34.641 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76199\n",
      "INFO:tensorflow:loss = 54.490616, step = 1765 (36.206 sec)\n",
      "INFO:tensorflow:nll_loss = 51.763042, kl_w = 0.11304584, kl_loss = 24.12803 (36.206 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.74562\n",
      "INFO:tensorflow:loss = 56.456055, step = 1865 (36.421 sec)\n",
      "INFO:tensorflow:nll_loss = 53.413612, kl_w = 0.13086486, kl_loss = 23.248743 (36.421 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.80187\n",
      "INFO:tensorflow:loss = 54.03767, step = 1965 (35.690 sec)\n",
      "INFO:tensorflow:nll_loss = 50.655396, kl_w = 0.15101443, kl_loss = 22.397024 (35.690 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.82004\n",
      "INFO:tensorflow:loss = 56.813725, step = 2065 (35.460 sec)\n",
      "INFO:tensorflow:nll_loss = 52.95685, kl_w = 0.17364664, kl_loss = 22.211058 (35.461 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.71987\n",
      "INFO:tensorflow:loss = 51.30645, step = 2165 (36.766 sec)\n",
      "INFO:tensorflow:nll_loss = 47.03791, kl_w = 0.19887616, kl_loss = 21.463306 (36.767 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:global_step/sec: 2.80633\n",
      "INFO:tensorflow:loss = 51.49467, step = 2265 (35.634 sec)\n",
      "INFO:tensorflow:nll_loss = 46.734463, kl_w = 0.22676536, kl_loss = 20.991776 (35.634 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 2346 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 52.414867.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-2346\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i love this film br i just like my movies in this film in <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a few years that was kohut of me it was going <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-2346\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 2347 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 57.95961, step = 2347\n",
      "INFO:tensorflow:nll_loss = 52.755302, kl_w = 0.25161827, kl_loss = 20.68335\n",
      "INFO:tensorflow:global_step/sec: 2.72904\n",
      "INFO:tensorflow:loss = 51.654736, step = 2447 (36.644 sec)\n",
      "INFO:tensorflow:nll_loss = 46.102142, kl_w = 0.2842792, kl_loss = 19.532188 (36.644 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.81022\n",
      "INFO:tensorflow:loss = 50.380527, step = 2547 (35.584 sec)\n",
      "INFO:tensorflow:nll_loss = 44.330475, kl_w = 0.31937042, kl_loss = 18.943687 (35.584 sec)\n",
      "INFO:tensorflow:global_step/sec: 3.01196\n",
      "INFO:tensorflow:loss = 51.621048, step = 2647 (33.202 sec)\n",
      "INFO:tensorflow:nll_loss = 44.849937, kl_w = 0.35663486, kl_loss = 18.98611 (33.203 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.7835\n",
      "INFO:tensorflow:loss = 52.455627, step = 2747 (35.925 sec)\n",
      "INFO:tensorflow:nll_loss = 45.4671, kl_w = 0.39571938, kl_loss = 17.660313 (35.925 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.77908\n",
      "INFO:tensorflow:loss = 58.247326, step = 2847 (35.983 sec)\n",
      "INFO:tensorflow:nll_loss = 51.40515, kl_w = 0.43618327, kl_loss = 15.686463 (35.983 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8843\n",
      "INFO:tensorflow:loss = 58.19124, step = 2947 (34.670 sec)\n",
      "INFO:tensorflow:nll_loss = 51.085724, kl_w = 0.47751516, kl_loss = 14.880184 (34.670 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8445\n",
      "INFO:tensorflow:loss = 56.006447, step = 3047 (35.156 sec)\n",
      "INFO:tensorflow:nll_loss = 48.35464, kl_w = 0.5191573, kl_loss = 14.7389 (35.156 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3128 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 60.438046.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-3128\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: ghoulies so bad and it was so much like it in the film and <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: it is a good movie with me better than it ghoulies ghoulies 10 it <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-3128\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 3129 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 55.659508, step = 3129\n",
      "INFO:tensorflow:nll_loss = 47.885826, kl_w = 0.55313194, kl_loss = 14.053941\n",
      "INFO:tensorflow:global_step/sec: 2.87395\n",
      "INFO:tensorflow:loss = 56.23067, step = 3229 (34.797 sec)\n",
      "INFO:tensorflow:nll_loss = 48.224976, kl_w = 0.5938731, kl_loss = 13.480481 (34.798 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8978\n",
      "INFO:tensorflow:loss = 57.249836, step = 3329 (34.508 sec)\n",
      "INFO:tensorflow:nll_loss = 49.10434, kl_w = 0.6333619, kl_loss = 12.860728 (34.508 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.97415\n",
      "INFO:tensorflow:loss = 57.842903, step = 3429 (33.623 sec)\n",
      "INFO:tensorflow:nll_loss = 49.232082, kl_w = 0.6711373, kl_loss = 12.830196 (33.623 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.80307\n",
      "INFO:tensorflow:loss = 55.76758, step = 3529 (35.675 sec)\n",
      "INFO:tensorflow:nll_loss = 47.091187, kl_w = 0.7068222, kl_loss = 12.275209 (35.675 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.97699\n",
      "INFO:tensorflow:loss = 55.25953, step = 3629 (33.591 sec)\n",
      "INFO:tensorflow:nll_loss = 46.753574, kl_w = 0.7401343, kl_loss = 11.492449 (33.591 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89707\n",
      "INFO:tensorflow:loss = 59.573807, step = 3729 (34.518 sec)\n",
      "INFO:tensorflow:nll_loss = 51.12761, kl_w = 0.7708882, kl_loss = 10.956448 (34.519 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76218\n",
      "INFO:tensorflow:loss = 62.0783, step = 3829 (36.204 sec)\n",
      "INFO:tensorflow:nll_loss = 54.342537, kl_w = 0.79899096, kl_loss = 9.681918 (36.204 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 3910 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 59.0858.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-3910\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: it is not worth seeing this film and it was br br the movie <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is a must see for the movie i think it is kohut <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-3910\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 3911 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 56.890656, step = 3911\n",
      "INFO:tensorflow:nll_loss = 48.960804, kl_w = 0.82004714, kl_loss = 9.669993\n",
      "INFO:tensorflow:global_step/sec: 2.75576\n",
      "INFO:tensorflow:loss = 53.243034, step = 4011 (36.290 sec)\n",
      "INFO:tensorflow:nll_loss = 45.423275, kl_w = 0.84334546, kl_loss = 9.272308 (36.289 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8747\n",
      "INFO:tensorflow:loss = 57.997505, step = 4111 (34.786 sec)\n",
      "INFO:tensorflow:nll_loss = 50.872063, kl_w = 0.8641271, kl_loss = 8.245829 (34.785 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.73175\n",
      "INFO:tensorflow:loss = 59.74556, step = 4211 (36.607 sec)\n",
      "INFO:tensorflow:nll_loss = 52.558067, kl_w = 0.88253593, kl_loss = 8.144134 (36.608 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8243\n",
      "INFO:tensorflow:loss = 61.985756, step = 4311 (35.407 sec)\n",
      "INFO:tensorflow:nll_loss = 55.173553, kl_w = 0.89874285, kl_loss = 7.579699 (35.406 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8366\n",
      "INFO:tensorflow:loss = 57.662785, step = 4411 (35.253 sec)\n",
      "INFO:tensorflow:nll_loss = 50.7009, kl_w = 0.9129343, kl_loss = 7.6258326 (35.254 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89177\n",
      "INFO:tensorflow:loss = 56.46083, step = 4511 (34.581 sec)\n",
      "INFO:tensorflow:nll_loss = 49.950287, kl_w = 0.92530197, kl_loss = 7.036128 (34.581 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.85604\n",
      "INFO:tensorflow:loss = 57.093227, step = 4611 (35.014 sec)\n",
      "INFO:tensorflow:nll_loss = 50.457706, kl_w = 0.93603605, kl_loss = 7.0889597 (35.014 sec)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Saving checkpoints for 4692 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 55.173218.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-4692\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i have seen and i love this movie br br this is one of <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie was a few times and i was very disappointed by i saw <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-4692\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 4693 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 51.711384, step = 4693\n",
      "INFO:tensorflow:nll_loss = 45.413948, kl_w = 0.94374704, kl_loss = 6.672802\n",
      "INFO:tensorflow:global_step/sec: 2.66921\n",
      "INFO:tensorflow:loss = 57.013412, step = 4793 (37.466 sec)\n",
      "INFO:tensorflow:nll_loss = 50.649048, kl_w = 0.95196813, kl_loss = 6.685483 (37.465 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.76638\n",
      "INFO:tensorflow:loss = 58.35082, step = 4893 (36.149 sec)\n",
      "INFO:tensorflow:nll_loss = 52.53527, kl_w = 0.9590399, kl_loss = 6.0639257 (36.149 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.90022\n",
      "INFO:tensorflow:loss = 58.723717, step = 4993 (34.480 sec)\n",
      "INFO:tensorflow:nll_loss = 53.046196, kl_w = 0.9651086, kl_loss = 5.88278 (34.480 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.78824\n",
      "INFO:tensorflow:loss = 58.9752, step = 5093 (35.865 sec)\n",
      "INFO:tensorflow:nll_loss = 53.410774, kl_w = 0.97030604, kl_loss = 5.7347136 (35.866 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.80598\n",
      "INFO:tensorflow:loss = 57.238068, step = 5193 (35.638 sec)\n",
      "INFO:tensorflow:nll_loss = 52.053337, kl_w = 0.97474945, kl_loss = 5.3190413 (35.637 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.85635\n",
      "INFO:tensorflow:loss = 55.83658, step = 5293 (35.010 sec)\n",
      "INFO:tensorflow:nll_loss = 50.51916, kl_w = 0.9785427, kl_loss = 5.4340167 (35.009 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.87689\n",
      "INFO:tensorflow:loss = 54.39757, step = 5393 (34.760 sec)\n",
      "INFO:tensorflow:nll_loss = 48.96374, kl_w = 0.9817768, kl_loss = 5.534689 (34.760 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 5474 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 50.892418.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-5474\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i think that this movie is a very good movie but it is the <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a very good movie that i have seen the first time and <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-5474\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 5475 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 53.94219, step = 5475\n",
      "INFO:tensorflow:nll_loss = 48.60081, kl_w = 0.98406756, kl_loss = 5.427856\n",
      "INFO:tensorflow:global_step/sec: 3.05952\n",
      "INFO:tensorflow:loss = 55.83977, step = 5575 (32.686 sec)\n",
      "INFO:tensorflow:nll_loss = 50.482925, kl_w = 0.9864804, kl_loss = 5.430262 (32.686 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.85403\n",
      "INFO:tensorflow:loss = 56.846596, step = 5675 (35.038 sec)\n",
      "INFO:tensorflow:nll_loss = 52.159424, kl_w = 0.98853207, kl_loss = 4.7415495 (35.039 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.877\n",
      "INFO:tensorflow:loss = 56.31526, step = 5775 (34.758 sec)\n",
      "INFO:tensorflow:nll_loss = 51.694088, kl_w = 0.9902755, kl_loss = 4.666553 (34.758 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.92142\n",
      "INFO:tensorflow:loss = 56.685272, step = 5875 (34.230 sec)\n",
      "INFO:tensorflow:nll_loss = 51.87616, kl_w = 0.9917561, kl_loss = 4.849087 (34.230 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89167\n",
      "INFO:tensorflow:loss = 55.321198, step = 5975 (34.582 sec)\n",
      "INFO:tensorflow:nll_loss = 50.347507, kl_w = 0.99301285, kl_loss = 5.008687 (34.582 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.8508\n",
      "INFO:tensorflow:loss = 53.734837, step = 6075 (35.078 sec)\n",
      "INFO:tensorflow:nll_loss = 48.86883, kl_w = 0.9940791, kl_loss = 4.8949876 (35.078 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.79324\n",
      "INFO:tensorflow:loss = 55.19273, step = 6175 (35.801 sec)\n",
      "INFO:tensorflow:nll_loss = 50.079063, kl_w = 0.99498355, kl_loss = 5.139447 (35.801 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 6256 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 51.023.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-6256\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: i really enjoyed this movie and i have to say about the movie and <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this movie is one of the most boring movies i've ever seen it is <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-6256\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 6257 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 51.67713, step = 6257\n",
      "INFO:tensorflow:nll_loss = 46.85193, kl_w = 0.9956215, kl_loss = 4.846423\n",
      "INFO:tensorflow:global_step/sec: 2.71833\n",
      "INFO:tensorflow:loss = 54.37004, step = 6357 (36.789 sec)\n",
      "INFO:tensorflow:nll_loss = 50.152927, kl_w = 0.9962913, kl_loss = 4.232812 (36.789 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.84951\n",
      "INFO:tensorflow:loss = 52.68529, step = 6457 (35.094 sec)\n",
      "INFO:tensorflow:nll_loss = 48.416943, kl_w = 0.99685884, kl_loss = 4.281799 (35.094 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.89514\n",
      "INFO:tensorflow:loss = 53.910164, step = 6557 (34.541 sec)\n",
      "INFO:tensorflow:nll_loss = 49.26461, kl_w = 0.9973398, kl_loss = 4.6579432 (34.540 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.91862\n",
      "INFO:tensorflow:loss = 53.67481, step = 6657 (34.263 sec)\n",
      "INFO:tensorflow:nll_loss = 49.328545, kl_w = 0.99774724, kl_loss = 4.356078 (34.263 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.97616\n",
      "INFO:tensorflow:loss = 50.009514, step = 6757 (33.600 sec)\n",
      "INFO:tensorflow:nll_loss = 45.334915, kl_w = 0.99809235, kl_loss = 4.6835337 (33.600 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.98461\n",
      "INFO:tensorflow:loss = 50.3114, step = 6857 (33.505 sec)\n",
      "INFO:tensorflow:nll_loss = 45.744537, kl_w = 0.99838483, kl_loss = 4.5742526 (33.506 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.93292\n",
      "INFO:tensorflow:loss = 51.031853, step = 6957 (34.096 sec)\n",
      "INFO:tensorflow:nll_loss = 46.42398, kl_w = 0.9986324, kl_loss = 4.614184 (34.096 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7038 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Loss for final step: 51.088085.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-7038\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: br br br br br this movie is bad br br the acting is <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is a very good movie that i was very excited to see the <end>\n",
      "\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-7038\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 7039 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:loss = 55.27629, step = 7039\n",
      "INFO:tensorflow:nll_loss = 50.594955, kl_w = 0.99880695, kl_loss = 4.6869264\n",
      "INFO:tensorflow:global_step/sec: 2.8692\n",
      "INFO:tensorflow:loss = 50.989983, step = 7139 (34.854 sec)\n",
      "INFO:tensorflow:nll_loss = 46.338837, kl_w = 0.9989899, kl_loss = 4.6558495 (34.854 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.71002\n",
      "INFO:tensorflow:loss = 47.334732, step = 7239 (36.900 sec)\n",
      "INFO:tensorflow:nll_loss = 42.98245, kl_w = 0.9991448, kl_loss = 4.3560066 (36.901 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.91693\n",
      "INFO:tensorflow:loss = 48.58064, step = 7339 (34.282 sec)\n",
      "INFO:tensorflow:nll_loss = 44.21469, kl_w = 0.999276, kl_loss = 4.369112 (34.282 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.83087\n",
      "INFO:tensorflow:loss = 49.905148, step = 7439 (35.325 sec)\n",
      "INFO:tensorflow:nll_loss = 45.666466, kl_w = 0.999387, kl_loss = 4.241282 (35.325 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.81586\n",
      "INFO:tensorflow:loss = 53.322124, step = 7539 (35.513 sec)\n",
      "INFO:tensorflow:nll_loss = 49.41973, kl_w = 0.99948114, kl_loss = 3.904418 (35.513 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.85661\n",
      "INFO:tensorflow:loss = 53.47346, step = 7639 (35.007 sec)\n",
      "INFO:tensorflow:nll_loss = 49.631226, kl_w = 0.9995608, kl_loss = 3.843925 (35.008 sec)\n",
      "INFO:tensorflow:global_step/sec: 2.87039\n",
      "INFO:tensorflow:loss = 54.642303, step = 7739 (34.838 sec)\n",
      "INFO:tensorflow:nll_loss = 50.58655, kl_w = 0.9996282, kl_loss = 4.0572624 (34.838 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 7820 into /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 54.56311.\n",
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/sx/fv0r97j96fz8njp14dt5g7940000gn/T/tmp07u6s_3s/model.ckpt-7820\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "\n",
      "Original: i love this film and i think it is one of the best films\n",
      "Reconstr: this movie is worth watching br br 4 out of 10 for violence and <end>\n",
      "\n",
      "Original: this movie is a waste of time and there is no point to watch it\n",
      "Reconstr: this is the worst movie i've seen in a long time i would recommend <end>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def inf_inp(test_strs):\n",
    "    x = [[PARAMS['word2idx'].get(w, 2) for w in s.split()] for s in test_strs]\n",
    "    x = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "        x, PARAMS['max_len'], truncating='post', padding='post')\n",
    "    return x\n",
    "\n",
    "def demo(test_strs, pred_ids):\n",
    "    for s, pred in zip(test_strs, pred_ids):\n",
    "        print('\\nOriginal:', s)\n",
    "        print('Reconstr:', ' '.join([PARAMS['idx2word'].get(idx, '<unk>') for idx in pred]))\n",
    "\n",
    "\n",
    "test_strs = ['i love this film and i think it is one of the best films',\n",
    "             'this movie is a waste of time and there is no point to watch it']\n",
    "\n",
    "estimator = tf.estimator.Estimator(model_fn)\n",
    "\n",
    "for _ in range(PARAMS['n_epochs']):\n",
    "    estimator.train(tf.estimator.inputs.numpy_input_fn(\n",
    "        x = enc_inp,\n",
    "        y = {'dec_inp': dec_inp, 'dec_out': dec_out},\n",
    "        batch_size = PARAMS['batch_size'],\n",
    "        shuffle = True))\n",
    "    \n",
    "    pred_ids = list(estimator.predict(tf.estimator.inputs.numpy_input_fn(\n",
    "        x = inf_inp(test_strs),\n",
    "        shuffle = False)))\n",
    "\n",
    "    demo(test_strs, pred_ids)\n",
    "    \n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
